'''
Created on Jan 17, 2010

@author: roman
'''

from google.appengine.ext import db

import logging
import simplejson
import twitter

import re

import urllib2 as u2

import pprint

from datetime import datetime
import time

from model.lasttwitterid import LastTwitterId
from model.twitterstatus import TwitterStatus 

import urlextractor


    
findUrlsRe = re.compile('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&#+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')

class RedirectHandler(u2.HTTPRedirectHandler):
    def __init__(self, reporter):
        self.reporter = reporter 

    def redirect_request(self, req, fp, code, msg, hdrs, newurl):
        logging.debug('redirect_request %s -- %s' % (msg, newurl))
        self.reporter.setUrl(newurl)
        raise u2.HTTPError(newurl, 0, msg, hdrs, fp)

class Reporter:
    def setUrl(self, url):
        self.url = url

def _unshortUrl(shortUrl):
    logging.debug("Unshortening URL %s" % shortUrl)
    r = Reporter()
    opener = u2.build_opener(RedirectHandler(r))
    
    fp = None
    
    try:
        opener.open(shortUrl)
    except u2.HTTPError, e:
        print(e)
        return r.url
    
    return shortUrl

def _parse_datetime(str):
	# We must parse datetime this way to work in python 2.4
	return datetime(*(time.strptime(str, '%a %b %d %H:%M:%S +0000 %Y')[0:6]))

def checkUsers():
    users = ['sigizmund', 'hd42', 'vicchi', 'siddhuw']
    
    allLastIds = LastTwitterId.all()
    
    # first we need to make sure that every user indeed has a valid LastTwitterId
    
    userToLastId = dict()
        
    for lastId in allLastIds:
        userToLastId[str(lastId.user)] = lastId
        if str(lastId.user.lower()) in users:
            users.remove(str(lastId.user.lower()))
    
    for u in users:
        logging.debug("No lastId: %s" % u)
    
    for u in users:
        lastId = LastTwitterId(user = u.lower(), statusId = '1')
        lastId.save()
        userToLastId[u] = lastId
    
    # by now every user has got a valid last Tweet ID

def checkUserUpdates(lastUpdateKey):
    class RedirectHandler(u2.HTTPRedirectHandler):
        pass
    
    
    opener = u2.build_opener()
    
    lastId = LastTwitterId.get(lastUpdateKey)
    
    logging.debug("Fetching feed for user %s, last time obtained at %s" % (lastId.user, lastId.obtainedAt))

    timeline = twitter.UserTimeline(username = str(lastId.user.lower()), since_id = str(lastId.statusId))
    updates = None
    
    try:
        updates = timeline.getLastUpdates()
    except:
        return # next time maybe

    logging.debug("I received %d updates for user %s" % (len(updates), lastId.user))

    for u in updates:
        
        if len(TwitterStatus.all().filter('statusId = ', str(u['id'])).fetch(1)) > 0:
            continue
        
        objectFormatted = pprint.pformat(u)
        logging.debug("Next status is: \n %s" % objectFormatted )
        
        # logging.debug(u)
        status = TwitterStatus(originalText = u['text'], user = lastId.user.lower(), statusId = str(u['id']), json = simplejson.dumps(u))
        status.createdAt = _parse_datetime(u['created_at'])
    
        # status.urls = map( lambda x : opener.open(x).url, findUrlsRe.findall(u['text']))
    
        shortUrls = urlextractor.extract(u['text'])
        #findUrlsRe.findall(u['text'])
    
        urls = []
    
        for su in shortUrls:
        
            logging.debug("Trying short URL %s" % su )
        
            try:
                longUrl = _unshortUrl(su)
                logging.debug("Resolved to long URL %s" % longUrl )
                urls.append(longUrl)
            except Exception, e:
                logging.error(e)
                urls.append(su)
    
        status.urls = urls
    
        logging.debug(urls)
    
        status.save()

    if len(updates) > 0:
        lastId.statusId = str(updates[0]['id'])
        lastId.save()
    

def readFeed():
    # let's ensure we have lastId for all our users
    checkUsers()
    
    # now let's go and check them
    
    q = LastTwitterId.all()
    q.order('-obtainedAt')
    
    
    
    # we always take top ten which were not updated for longer 
    results = q.fetch(10)
    
    for lastId in results:
        checkUserUpdates(lastId.key())

#def processBackLog(username):
#    q = TwitterStatus.all()
#    q.filter('username = ', username)
    

def fetchOldUpdates(handler):  
    username = 'sigizmund'     
    
    if len(handler.request.get('username')) > 1:
        username = handler.request.get('username')
    
    def callback(tweets):
        for u in tweets:
            q = TwitterStatus.all()
            q.filter('statusId = ', str(u['id']))
            
            if len(q.fetch(5)) > 0:
                continue

            shortUrls = findUrlsRe.findall(u['text'])
            if len(shortUrls) == 0:
                continue
            
            status = TwitterStatus(originalText = u['text'], user = username.lower(), statusId = str(u['id']), json = simplejson.dumps(u))
            status.createdAt = _parse_datetime(u['created_at'])
            


            urls = []

            for su in shortUrls:
                logging.debug("Trying short URL %s" % su )
                
                if su.find('yfrog') != -1:
                    urls.append(su)
                    continue

                try:
                    longUrl = _unshortUrl(su)
                    logging.debug("Resolved to long URL %s" % longUrl )
                    urls.append(longUrl)
                except:
                    continue

            status.urls = urls

            logging.debug(status)
            try:
                status.save()
            except UnicodeDecodeError, e:
                logging.error("Unicode issue")
                logging.error(e)
#                status.delete()
            
    
    timeline = twitter.UserTimeline(username = username)
    timeline.getOldUpdates(callback = callback)

def processBackLog():
    allLastIds = LastTwitterId.all()
    q.order('-obtainedAt')
    
    users = q.fetch(50)
    
    for u in users:
        user = str(u.user)
        qu = TwitterStatus.all()
        qu.filter('user = ', user)
        
        updates = qu.fetch()
        pass
        
        